{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-08T03:06:41.421996600Z",
     "start_time": "2024-05-08T03:06:39.095327500Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "# prefect match，reference是target,candidate是预测输出的\n",
    "reference = [['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog']]\n",
    "candidate = ['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog']\n",
    "score = sentence_bleu(reference, candidate)\n",
    "print(score)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function sentence_bleu in module nltk.translate.bleu_score:\n",
      "\n",
      "sentence_bleu(references, hypothesis, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=None, auto_reweigh=False)\n",
      "    Calculate BLEU score (Bilingual Evaluation Understudy) from\n",
      "    Papineni, Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002.\n",
      "    \"BLEU: a method for automatic evaluation of machine translation.\"\n",
      "    In Proceedings of ACL. http://www.aclweb.org/anthology/P02-1040.pdf\n",
      "    \n",
      "    >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',\n",
      "    ...               'ensures', 'that', 'the', 'military', 'always',\n",
      "    ...               'obeys', 'the', 'commands', 'of', 'the', 'party']\n",
      "    \n",
      "    >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops',\n",
      "    ...               'forever', 'hearing', 'the', 'activity', 'guidebook',\n",
      "    ...               'that', 'party', 'direct']\n",
      "    \n",
      "    >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',\n",
      "    ...               'ensures', 'that', 'the', 'military', 'will', 'forever',\n",
      "    ...               'heed', 'Party', 'commands']\n",
      "    \n",
      "    >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',\n",
      "    ...               'guarantees', 'the', 'military', 'forces', 'always',\n",
      "    ...               'being', 'under', 'the', 'command', 'of', 'the',\n",
      "    ...               'Party']\n",
      "    \n",
      "    >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',\n",
      "    ...               'army', 'always', 'to', 'heed', 'the', 'directions',\n",
      "    ...               'of', 'the', 'party']\n",
      "    \n",
      "    >>> sentence_bleu([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS\n",
      "    0.5045...\n",
      "    \n",
      "    If there is no ngrams overlap for any order of n-grams, BLEU returns the\n",
      "    value 0. This is because the precision for the order of n-grams without\n",
      "    overlap is 0, and the geometric mean in the final BLEU score computation\n",
      "    multiplies the 0 with the precision of other n-grams. This results in 0\n",
      "    (independently of the precision of the othe n-gram orders). The following\n",
      "    example has zero 3-gram and 4-gram overlaps:\n",
      "    \n",
      "    >>> round(sentence_bleu([reference1, reference2, reference3], hypothesis2),4) # doctest: +ELLIPSIS\n",
      "    0.0\n",
      "    \n",
      "    To avoid this harsh behaviour when no ngram overlaps are found a smoothing\n",
      "    function can be used.\n",
      "    \n",
      "    >>> chencherry = SmoothingFunction()\n",
      "    >>> sentence_bleu([reference1, reference2, reference3], hypothesis2,\n",
      "    ...     smoothing_function=chencherry.method1) # doctest: +ELLIPSIS\n",
      "    0.0370...\n",
      "    \n",
      "    The default BLEU calculates a score for up to 4-grams using uniform\n",
      "    weights (this is called BLEU-4). To evaluate your translations with\n",
      "    higher/lower order ngrams, use customized weights. E.g. when accounting\n",
      "    for up to 5-grams with uniform weights (this is called BLEU-5) use:\n",
      "    \n",
      "    >>> weights = (1./5., 1./5., 1./5., 1./5., 1./5.)\n",
      "    >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS\n",
      "    0.3920...\n",
      "    \n",
      "    :param references: reference sentences\n",
      "    :type references: list(list(str))\n",
      "    :param hypothesis: a hypothesis sentence\n",
      "    :type hypothesis: list(str)\n",
      "    :param weights: weights for unigrams, bigrams, trigrams and so on\n",
      "    :type weights: list(float)\n",
      "    :param smoothing_function:\n",
      "    :type smoothing_function: SmoothingFunction\n",
      "    :param auto_reweigh: Option to re-normalize the weights uniformly.\n",
      "    :type auto_reweigh: bool\n",
      "    :return: The sentence-level BLEU score.\n",
      "    :rtype: float\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(sentence_bleu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-08T03:12:18.591502600Z",
     "start_time": "2024-05-08T03:12:18.561518200Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.816496580927726\n"
     ]
    }
   ],
   "source": [
    "# one word different,默认是4-gram\n",
    "candidate = ['the', 'fast', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog']\n",
    "# score = sentence_bleu(reference, candidate,weights=(0.25, 0.25, 0.25, 0.25))\n",
    "score = sentence_bleu(reference, candidate,weights=(0.5, 0.5, 0, 0))\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-08T03:11:09.194594Z",
     "start_time": "2024-05-08T03:11:09.166607800Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "0.7506238537503395"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.exp(1/4*np.log(8/9)+1/4*np.log(6/8)+1/4*np.log(5/7)+1/4*np.log(4/6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8888888888888888"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(np.log(8/9))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-08T03:11:38.939816400Z",
     "start_time": "2024-05-08T03:11:38.899840900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-08T03:13:33.563159700Z",
     "start_time": "2024-05-08T03:13:33.542176100Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4854917717073234\n"
     ]
    }
   ],
   "source": [
    "# two words different\n",
    "\n",
    "candidate = ['the', 'fast', 'brown', 'fox', 'jumped', 'over', 'the', 'sleepy', 'dog']\n",
    "score = sentence_bleu(reference, candidate)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-08T03:14:05.411295600Z",
     "start_time": "2024-05-08T03:14:05.402291400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4835447404743731\n"
     ]
    }
   ],
   "source": [
    "# 删除candidate句子最后两个单词, 即让candidate变短，缺失词需要考虑精确度\n",
    "\n",
    "candidate = ['the', 'fast', 'brown', 'fox', 'jumped', 'over', 'the']\n",
    "score = sentence_bleu(reference, candidate)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49228386893821674"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.exp(1/4*np.log(5/7)+1/4*np.log(4/6)+1/4*np.log(3/5)+1/4*np.log(2/4))*np.exp(-2/9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-08T03:14:23.327520900Z",
     "start_time": "2024-05-08T03:14:23.295538400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7860753021519787\n"
     ]
    }
   ],
   "source": [
    "#多两个词\n",
    "candidate = ['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dog', 'from', 'space']\n",
    "score = sentence_bleu(reference, candidate)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0547686614863434e-154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luke/.virtualenvs/tf_py3/lib/python3.6/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 3-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/home/luke/.virtualenvs/tf_py3/lib/python3.6/site-packages/nltk/translate/bleu_score.py:516: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4-gram cumulative BLEU\n",
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "\n",
    "reference = [['this', 'is', 'small', 'test']]\n",
    "candidate = ['this', 'is', 'a', 'test']\n",
    "score = sentence_bleu(reference, candidate)\n",
    "print(score)\n",
    "score = sentence_bleu(reference, candidate, weights=(1, 0, 0, 0))\n",
    "score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luke/.virtualenvs/tf_py3/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: divide by zero encountered in log\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.exp(1/4*np.log(3/4)+1/4*np.log(1/3)+1/4*np.log(0)+1/4*np.log(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0547686614863434e-154"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cumulative 1-gram: 0.750000\n",
      "Cumulative 2-gram: 0.500000\n",
      "Cumulative 3-gram: 0.000000\n",
      "Cumulative 4-gram: 0.000000\n"
     ]
    }
   ],
   "source": [
    "print('Cumulative 1-gram: %f' % sentence_bleu(reference, candidate, weights=(1, 0, 0, 0)))\n",
    "\n",
    "print('Cumulative 2-gram: %f' % sentence_bleu(reference, candidate, weights=(0.5, 0.5, 0, 0)))\n",
    "\n",
    "print('Cumulative 3-gram: %f' % sentence_bleu(reference, candidate, weights=(0.33, 0.33, 0.33, 0)))\n",
    "\n",
    "print('Cumulative 4-gram: %f' % sentence_bleu(reference, candidate, weights=(0.25, 0.25, 0.25, 0.25)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49749371855331004"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#这是2-gram的计算方法\n",
    "import numpy as np\n",
    "np.exp(1/2*np.log(0.75)+1/2*np.log(0.33))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luke/.virtualenvs/tf_py3/lib/python3.6/site-packages/ipykernel_launcher.py:1: RuntimeWarning: divide by zero encountered in log\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-inf"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luke/.virtualenvs/tf_py3/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: divide by zero encountered in log\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.exp(1/3*np.log(0.75)+1/3*np.log(0.33)+1/3*np.log(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7825422900366437"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reference = [[i for i in range(10)]]\n",
    "candidate = [0, 1, 2, 3, 4, 5, 6, 7, 5, 9]\n",
    "score = sentence_bleu(reference, candidate)\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# from torchtext.data.metrics import bleu_score"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
